#基于嵌套Pipeline和FeatureUnion复合数据工作流的营销响应预测
import time
import numpy as np
import pandas as pd
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.ensemble import RandomForestClassifier,ExtraTreesClassifier
from sklearn.feature_selection import RFE
from sklearn.model_selection import cross_val_score,StratifiedKFold
from sklearn.pipeline import Pipeline,FeatureUnion
from imblearn.over_sampling import SMOTE#过抽样处理库SMOTE
from sklearn.metrics import f1_score,accuracy_score,precision_score

#读取数据
rawdata=pd.read_excel('order.xlsx',sheet_name=0)

#数据审查
##查看基本状态
#print('records:{0} features:{1}'.format(rawdata.shape[0],rawdata.shape[1]))
#
##查看缺失值
#print('NaN records count:',rawdata.isnull().any(axis=1).count())#具有缺失值的记录数
#na_cols=rawdata.isnull().any()
#print('NaN cols',na_cols[na_cols.values==True])
#
##查看样本均衡情况
#print('samples distribution:',rawdata['value_level'].groupby(rawdata['response']).count())


#数据预处理
#空值的处理
na_rules={'age':rawdata['age'].mean(),
          'total_pageviews':rawdata['total_pageviews'].mean(),
          'edu':rawdata['edu'].median(),
          'edu_ages':rawdata['edu_ages'].median(),
          'user_level':rawdata['user_level'].median(),
          'industry':rawdata['industry'].median(),
          'act_level':rawdata['act_level'].median(),
          'sex':rawdata['sex'].median(),
          'red_money':rawdata['red_money'].median(),
          'region':rawdata['region'].median()
          }
#填充数据
rawdata=rawdata.fillna(na_rules)

#分割特征和目标
num=int(0.7*rawdata.shape[0])
X,y=rawdata.drop('response',axis=1),rawdata['response']
x_train,x_test=X.iloc[:num,:],X.iloc[num:,:]
y_train,y_test=y.iloc[:num],y.iloc[num:]


#样本均衡
#建立SMOTTE模型对象
model_smote=SMOTE()
#输入数据并做过抽样处理
x_smote_resampled,y_smote_resampled=model_smote.fit_sample(x_train,y_train)

#模型训练
model_etc=ExtraTreesClassifier()#------用于EFE的模型对象
model_rfe=RFE(model_etc)#--------------使用RFE方法提取重要特征
model_lda=LinearDiscriminantAnalysis()#---------LDA模型对象
model_rf=RandomForestClassifier()#------------分类对象

# 构建带有嵌套的pipeline
pipelines = Pipeline([
    ('feature_union', FeatureUnion(  # 组合特征pipeline
        transformer_list=[
            ('model_rfe', model_rfe),  # 通过RFE中提取特征
            ('model_lda', model_lda),  # 通过LDA提取特征
        ],
        transformer_weights={  # 建立不同特征模型的权重
            'model_rfe': 1,  # RFE模型权重
            'model_lda': 0.8,  # LDA模型权重
        },
    )),
    ('model_rf', model_rf),  # rf模型对象
])


# 设置pipe参数值
pipelines.set_params(feature_union__model_rfe__estimator__n_estimators=20)  # ExtraTreesClassifier中n_estimators值
pipelines.set_params(feature_union__model_rfe__estimator__n_jobs=-1)  # ExtraTreesClassifier中n_jobs值
pipelines.set_params(feature_union__model_rfe__n_features_to_select=20)  # RFE中n_features_to_select值
pipelines.set_params(feature_union__model_lda__n_components=1)  # LDA中n_components值
pipelines.set_params(feature_union__n_jobs=-1)  # FeatureUnion中n_jobs值
# project_pipeline.get_params()  # 打印pipline参数详情



# pipeline交叉检验
cv = StratifiedKFold(3)  # 设置交叉检验
score_list = list()  # 建立空列表，用于存放交叉检验得分
time_list = list()  # 建立空列表，用于存储时间
n_estimators = [10, 50, 100]  # 设置pipeline中rf的n_estimators值域
for parameter in n_estimators:  # 遍历每个参数值
    t1 = time.time()  # 记录交叉检验开始的时间
    print('set parameters: %s' % parameter)  # 打印当前模型使用的参数
    pipelines.set_params(model_rf__n_estimators=parameter)  # 通过管道设置分类模型参数
    score_tmp = cross_val_score(pipelines, x_train, y_train, scoring='accuracy',cv=cv,n_jobs=-1)  # 使用交叉检验计算得分
    time_list.append(time.time() - t1)  # 计算交叉检验时间并追加到列表
    score_list.append(score_tmp)  # 将得分追加到列表


# 组合交叉检验得分和详情数据
time_pd = pd.DataFrame.from_dict({'n_estimators':n_estimators,'time':time_list})
score_pd = pd.DataFrame(score_list,columns=[''.join(['score', str(i+1)]) for i in range(len(score_list))])
pd_merge = pd.concat((time_pd,score_pd),axis=1) # 完整数据框
pd_merge['score_mean'] = pd_merge.iloc[:, 2:-1].mean(axis=1)  # 计算得分均值
pd_merge['score_std'] = pd_merge.iloc[:, 2:-2].std(axis=1)  # 计算得分std
print(pd_merge.head())

# 将最优参数设置到模型中，并训练pipeline
pipelines.set_params(model_rf__n_estimators=50)  # 设置最优参数值
pipelines.fit(x_train, y_train)  # 训练pipeline模型


# 模型效果检验
pre_test = pipelines.predict(x_test)
scores = [i(y_test,pre_test) for i in [f1_score,accuracy_score,precision_score]]
print('socres result: f1:{0},accuracy:{1},precision:{2}'.format(scores[0],scores[1],scores[2]))


# 读取新数据集
new_data = pd.read_excel('order.xlsx', sheet_name=1)  # 读取要预测的数据集


# 查看基本状态
print('records:{0} features:{1}'.format(new_data.shape[0], (new_data.shape[1])))  # 打印数据集形状
# 缺失值处理
print('NaN records count:',new_data.isnull().any(axis=1).count())  # 具有缺失值的记录数

# 填充NA值
new_data_fillna = new_data.fillna(na_rules)  # 使用指定方法填充缺失值


# 预测概率
pre_labels = pd.DataFrame(pipelines.predict(new_data_fillna), columns=['labels'])  # 获得预测标签
pre_pro = pd.DataFrame(pipelines.predict_proba(new_data_fillna), columns=['pro1', 'pro2'])  # 获得预测概率
predict_pd = pd.concat((pre_labels, pre_pro), axis=1)  # 合并数据
print(predict_pd.head())  # 打印前2条结果


writer = pd.ExcelWriter('order_predict_result.xlsx')  # 创建写入文件对象
predict_pd.to_excel(writer, 'Sheet1')  # 将数据写入sheet1
writer.save()
writer.close()







